# -*- coding: utf-8 -*-
import scrapy
from selenium import webdriver
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
import scrapy
import datetime
from scrapy.http import Request
from urllib import parse
from scrapy.loader import ItemLoader
from myayi.items import MyayiItem
from scrapy.contrib.linkextractors import LinkExtractor
from myayi.unity.chinese_to_arabic import chinese_to_arabic 
from myayi.unity.getnumber import get_number

class MayiSpider(scrapy.Spider):
    name = "mayi"
    allowed_domains = ["www.ishuyin.com"]
    start_urls = ['https://www.ishuyin.com/show.php?id=19567#liebiao']
   
    def __init__(self,**kwargs):
        self.options = webdriver.ChromeOptions()
        self.options.add_argument('incognito')#隐私窗口
        self.options.add_argument('disable-infobars')
        prefs={"profile.managed_default_content_settings.images":2}
        self.options.add_experimental_option("prefs",prefs)
        self.options.add_argument("--mute-audio")
        self.options.add_argument('-headless')
        self.options.add_argument('window-size=940,738')
        self.browser = webdriver.Chrome(chrome_options=self.options)
        self.browser.set_page_load_timeout(50)
        self.browser.set_script_timeout(50)
        super(MayiSpider, self).__init__()
        dispatcher.connect(self.spider_closed, signals.spider_closed)
    
    def start_requests(self):
        for url in self.start_urls:
            yield self.make_requests_from_url(url)
    def make_requests_from_url(self, url):
        return scrapy.Request(url, dont_filter=True, meta={'usedSelenium':True})
    def spider_closed(self, spider):
        #当爬虫退出的时候关闭chrome
        print ("spider closed")
        self.browser.quit()
    def parse(self, response):
        le = LinkExtractor(restrict_xpaths='//*[@id="articleDiv"]/div/div/div[1]/div[2]/a')
        links = le.extract_links(response)
        for link in links:
            # print(link)
            number = get_number(link)
            meta_ = {"number": "%s"%number,'usedSelenium':True}
            yield scrapy.Request(url=link.url,callback=self.parse_item,meta=meta_)
    def parse_item(self, response):
        mayiItem =  MyayiItem()
        number = response.meta.get('number')
        mayiItem['song_name'] = number 

        mayiItem['file_urls'] = [response.xpath('//*[@id="jp_audio_0"]/@src').extract_first()]
        return mayiItem
        
        
                
